
from sklearn.datasets import fetch_openml
from matplotlib import pyplot as plt
import keras
from threading import Thread

import seaborn as sns; 
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score

from sklearn.metrics import accuracy_score
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
from tensorflow.keras.utils import to_categorical
import numpy as np
import pandas as pd
import matplotlib.ticker as ticker


data = fetch_openml('Fashion-MNIST', as_frame=False, parser='auto')


list(data)

['data',
 'target',
 'frame',
 'categories',
 'feature_names',
 'target_names',
 'DESCR',
 'details',
 'url']


print(data.DESCR)

**Author**: Han Xiao, Kashif Rasul, Roland Vollgraf  
**Source**: [Zalando Research](https://github.com/zalandoresearch/fashion-mnist)  
**Please cite**: Han Xiao and Kashif Rasul and Roland Vollgraf, Fashion-MNIST: a Novel Image Dataset for Benchmarking Machine Learning Algorithms, arXiv, cs.LG/1708.07747  

Fashion-MNIST is a dataset of Zalando's article images, consisting of a training set of 60,000 examples and a test set of 10,000 examples. Each example is a 28x28 grayscale image, associated with a label from 10 classes. Fashion-MNIST is intended to serve as a direct drop-in replacement for the original MNIST dataset for benchmarking machine learning algorithms. It shares the same image size and structure of training and testing splits. 

Raw data available at: https://github.com/zalandoresearch/fashion-mnist

### Target classes
Each training and test example is assigned to one of the following labels:
Label  Description  
0  T-shirt/top  
1  Trouser  
2  Pullover  
3  Dress  
4  Coat  
5  Sandal  
6  Shirt  
7  Sneaker  
8  Bag  
9  Ankle boot

Downloaded from openml.org.


data.data.shape

(70000, 784)


data.target.shape

(70000,)


X = data.data
y = to_categorical(data.target)


def plot_image(image_data):
    image = image_data.reshape(28, 28)
    plt.imshow(image, cmap="binary")
    plt.axis("off")


plt.figure(figsize=(9, 9))
for idx, image_data in enumerate(X[:100]):
    plt.subplot(10, 10, idx + 1)
    plot_image(image_data)
plt.subplots_adjust(wspace=0, hspace=0)
plt.show()


X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]

X_train = X_train.reshape((60000, 28 * 28))  # flattening the 28x28 image into dim=1 (784,) vector (and over all images in the training set)
X_test = X_test.reshape((10000, 28 * 28))

X_train = X_train.astype("float32") / 255  # normalizing the data to the range [0,1] by min-max range, and casting it to float32
X_test = X_test.astype("float32") / 255


def fit_model(model, epochs, batch_size, verbose):
    model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
    history = model.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=verbose)
    return model, history


def one_hot_to_label(one_hot):
    label = list()

    for i in range(len(one_hot)):
        label.append(np.argmax(one_hot[i]))

    return label


def evaluate_model(y_test, y_pred, labels):
  y_pred = one_hot_to_label(y_pred)
  y_test = one_hot_to_label(y_test)
  cm = confusion_matrix(y_test, y_pred)
  cr = classification_report(y_test, y_pred, zero_division=0)

  
  sns.heatmap(cm.T, square=True, annot=True, fmt='d', xticklabels = labels, yticklabels = labels)
  plt.xlabel('true label')
  plt.ylabel('predicted label')
  print(cr)
  plt.show()
  return cm, cr


def evaluate_histories(histories, models, figsize):
    _,(ax1,ax2)=plt.subplots(2,figsize=figsize)

    ax1.xaxis.set_major_locator(ticker.MaxNLocator(integer=True))
    ax2.xaxis.set_major_locator(ticker.MaxNLocator(integer=True))

    for i in range(len(histories)):
        val_acc = histories[i].history['accuracy']
        val_loss = histories[i].history['loss']
        ax1.plot(val_acc, label=models[i])
        ax2.plot(val_loss, label=models[i])
    ax1.set_ylabel('Accuracy')
    ax2.set_ylabel('Loss')
    ax1.set_xlabel('Epochs')
    ax2.set_xlabel('Epochs')
    ax1.legend()
    ax2.legend()
    plt.show()


def classification_report_to_df(cr):
    cr_data = [x.split() for x in cr.split('\n')]
    cr_df = pd.DataFrame(columns=cr_data[0][:-1])
    for data in cr_data[2:12]:
        data_df = pd.DataFrame([data[1:-1]], columns=cr_data[0][:-1])   
        cr_df = pd.concat([cr_df, data_df], ignore_index=True)

    return(cr_df)


def evaluate_classification_reports(cr_array):
    df = pd.DataFrame()

    for i in range(len(cr_array)):
        cr_df = classification_report_to_df(cr_array[i])[3:]
        cr_df.index = cr_df.index + 1
        cr_df = cr_df.stack()
        cr_df.index = cr_df.index.map('{0[1]}_{0[0]}'.format)
        cr_df = cr_df.to_frame().T
        df = pd.concat([df, cr_df], ignore_index=True)

    object_columns = df.columns[df.dtypes.eq(object)]
    df[object_columns] = df[object_columns].apply(
        pd.to_numeric,
        errors='coerce',
        axis=0
    )

    df_min = pd.DataFrame(df).T
    df_max = pd.DataFrame(df.idxmax()).value_counts()

    best_model = df_max.T.idxmax()[0]
    stable_model = df_min.min().idxmax()

    return best_model, stable_model


def make_model(layers, input, output):
    model = Sequential()

    if len(layers) == 0:
        model.add(Dense(output, input_dim=input, activation='softmax'))

    else:
        model.add(Dense(layers[-1], input_dim=input, activation='relu'))
        
        for i in range(len(layers) - 1):
            model.add(Dense(i, activation='relu'))

        model.add(Dense(10, activation='softmax'))

    return model


def build_and_evaluate_model(epochs, batch_size, input, output, verbose, layers, result, index):
    model = make_model(layers, input, output)
    model, history = fit_model(model,epochs, batch_size, verbose)
    y_pred = model.predict(X_test, verbose=0)
    result[index] = {'history': history, 'y_test': y_test, 'y_pred': y_pred}


def build_and_evaluate_models(models):
    
    labels =  ["T-shirt/top", "Trouser", "Pullover", "Dress", "Coat", "Sandal", "Shirt", "Sneaker", "Bag", "Ankle boot"]
    epochs =  5
    batch_size =  128
    input =  784
    output =  10
    verbose =  0


    model_results = {
        'models': models,
        'history_array':[],
        'cm_array':[],
        'cr_array':[]
    }

    threads = [None] * len(models)
    results = [None] * len(models)

    for i in range(len(threads)):
        threads[i] = Thread(target=build_and_evaluate_model, args=(epochs, batch_size, input, output, verbose, models[i], results, i))
        threads[i].start()

    for i in range(len(threads)):
        threads[i].join()

    for i in range(len(results)):
        print(models[i])
        cm, cr = evaluate_model(y_test, results[i]['y_pred'], labels)

        model_results['history_array'].append(results[i]['history'])
        model_results['cm_array'].append(cm)
        model_results['cr_array'].append(cr)
        
    evaluate_histories(model_results['history_array'], models, (8, 100))
    return evaluate_classification_reports(model_results['cr_array'])


models = np.reshape(np.arange(10, 1000, 10), (-1, 1))

best_model, stable_model = build_and_evaluate_models(models)
print('Best model: ', models[best_model])
print('Stable model: ', models[stable_model])

[10]
              precision    recall  f1-score   support

           0       0.77      0.81      0.79      1000
           1       0.96      0.95      0.96      1000
           2       0.66      0.74      0.70      1000
           3       0.87      0.79      0.83      1000
           4       0.71      0.67      0.69      1000
           5       0.91      0.93      0.92      1000
           6       0.56      0.58      0.57      1000
           7       0.88      0.94      0.91      1000
           8       0.96      0.91      0.94      1000
           9       0.97      0.89      0.93      1000

    accuracy                           0.82     10000
   macro avg       0.83      0.82      0.82     10000
weighted avg       0.83      0.82      0.82     10000

[20]
              precision    recall  f1-score   support

           0       0.78      0.84      0.81      1000
           1       0.99      0.95      0.97      1000
           2       0.82      0.63      0.71      1000
           3       0.84      0.87      0.86      1000
           4       0.70      0.81      0.75      1000
           5       0.96      0.92      0.94      1000
           6       0.63      0.60      0.62      1000
           7       0.92      0.93      0.93      1000
           8       0.93      0.96      0.95      1000
           9       0.93      0.96      0.95      1000

    accuracy                           0.85     10000
   macro avg       0.85      0.85      0.85     10000
weighted avg       0.85      0.85      0.85     10000

[30]
              precision    recall  f1-score   support

           0       0.74      0.88      0.80      1000
           1       0.96      0.97      0.96      1000
           2       0.63      0.85      0.72      1000
           3       0.89      0.81      0.85      1000
           4       0.68      0.80      0.73      1000
           5       0.93      0.96      0.94      1000
           6       0.83      0.31      0.45      1000
           7       0.94      0.91      0.93      1000
           8       0.96      0.95      0.95      1000
           9       0.95      0.95      0.95      1000

    accuracy                           0.84     10000
   macro avg       0.85      0.84      0.83     10000
weighted avg       0.85      0.84      0.83     10000

[40]
              precision    recall  f1-score   support

           0       0.78      0.83      0.80      1000
           1       0.96      0.97      0.97      1000
           2       0.79      0.75      0.77      1000
           3       0.91      0.79      0.84      1000
           4       0.78      0.74      0.76      1000
           5       0.95      0.95      0.95      1000
           6       0.59      0.69      0.64      1000
           7       0.94      0.91      0.92      1000
           8       0.96      0.95      0.96      1000
           9       0.93      0.96      0.95      1000

    accuracy                           0.85     10000
   macro avg       0.86      0.85      0.86     10000
weighted avg       0.86      0.85      0.86     10000

[50]
              precision    recall  f1-score   support

           0       0.82      0.81      0.82      1000
           1       0.99      0.95      0.97      1000
           2       0.85      0.65      0.74      1000
           3       0.83      0.89      0.86      1000
           4       0.69      0.85      0.76      1000
           5       0.97      0.92      0.95      1000
           6       0.66      0.63      0.64      1000
           7       0.92      0.94      0.93      1000
           8       0.95      0.97      0.96      1000
           9       0.93      0.96      0.95      1000

    accuracy                           0.86     10000
   macro avg       0.86      0.86      0.86     10000
weighted avg       0.86      0.86      0.86     10000

[60]
              precision    recall  f1-score   support

           0       0.82      0.81      0.82      1000
           1       0.98      0.96      0.97      1000
           2       0.71      0.83      0.77      1000
           3       0.85      0.87      0.86      1000
           4       0.75      0.82      0.78      1000
           5       0.95      0.97      0.96      1000
           6       0.78      0.50      0.61      1000
           7       0.93      0.94      0.94      1000
           8       0.91      0.98      0.94      1000
           9       0.97      0.95      0.96      1000

    accuracy                           0.86     10000
   macro avg       0.86      0.86      0.86     10000
weighted avg       0.86      0.86      0.86     10000

[70]
              precision    recall  f1-score   support

           0       0.77      0.89      0.82      1000
           1       0.98      0.96      0.97      1000
           2       0.65      0.90      0.75      1000
           3       0.88      0.87      0.88      1000
           4       0.83      0.67      0.74      1000
           5       0.98      0.92      0.95      1000
           6       0.78      0.53      0.63      1000
           7       0.88      0.98      0.93      1000
           8       0.96      0.96      0.96      1000
           9       0.97      0.92      0.95      1000

    accuracy                           0.86     10000
   macro avg       0.87      0.86      0.86     10000
weighted avg       0.87      0.86      0.86     10000

[80]
              precision    recall  f1-score   support

           0       0.73      0.91      0.81      1000
           1       0.99      0.97      0.98      1000
           2       0.74      0.82      0.78      1000
           3       0.85      0.91      0.88      1000
           4       0.83      0.69      0.75      1000
           5       0.98      0.93      0.95      1000
           6       0.71      0.54      0.62      1000
           7       0.93      0.91      0.92      1000
           8       0.97      0.96      0.96      1000
           9       0.90      0.98      0.94      1000

    accuracy                           0.86     10000
   macro avg       0.86      0.86      0.86     10000
weighted avg       0.86      0.86      0.86     10000

[90]
              precision    recall  f1-score   support

           0       0.83      0.80      0.82      1000
           1       0.98      0.97      0.98      1000
           2       0.87      0.64      0.73      1000
           3       0.87      0.88      0.88      1000
           4       0.73      0.82      0.77      1000
           5       0.97      0.95      0.96      1000
           6       0.61      0.71      0.66      1000
           7       0.91      0.97      0.94      1000
           8       0.95      0.97      0.96      1000
           9       0.98      0.93      0.95      1000

    accuracy                           0.86     10000
   macro avg       0.87      0.86      0.86     10000
weighted avg       0.87      0.86      0.86     10000

[100]
              precision    recall  f1-score   support

           0       0.82      0.83      0.82      1000
           1       0.99      0.96      0.98      1000
           2       0.81      0.69      0.75      1000
           3       0.89      0.87      0.88      1000
           4       0.83      0.66      0.74      1000
           5       0.98      0.93      0.95      1000
           6       0.55      0.75      0.63      1000
           7       0.85      0.98      0.91      1000
           8       0.95      0.98      0.96      1000
           9       0.98      0.87      0.92      1000

    accuracy                           0.85     10000
   macro avg       0.86      0.85      0.85     10000
weighted avg       0.86      0.85      0.85     10000


# models = np.reshape(np.arange(10, 1000, 10), (-1, 1))

# best_model, stable_model = build_and_evaluate_models(models)
# print('Best model: ', models[best_model])
# print('Stable model: ', models[stable_model])


# models = np.reshape(np.arange(10, 1000, 10), (-1, 1))

# best_model, stable_model = build_and_evaluate_models(models)
# print('Best model: ', models[best_model])
# print('Stable model: ', models[stable_model])


# models = np.reshape(np.arange(10, 50, 1), (-1, 1))

# best_model, stable_model = build_and_evaluate_models([[], [10], [20]])
# print('Best model: ', models[best_model])
# print('Stable model: ', models[stable_model])

1. Using Keras Datasets, obtain the fMNIST data set. Explore and describe it as you would do with any new dataset that you start working with (2 points).¶

2. Train a dense neural network using Keras¶